# -*- coding: utf-8 -*-
import scrapy


class DescriptionSpider(scrapy.Spider):
    name = 'description'
    allowed_domains = ['newssc.org']
    start_urls = ['http://world.newssc.org/gj/']

    def parse(self, response):
        href_list=response.xpath('//div[@class="list-box"]/div[@class="list-items"]/ul/li/a/@href').extract()
        for hrefs in href_list:
            href = response.urljoin(hrefs)
            yield scrapy.Request(url=href,callback=self.getcontet, dont_filter=True)

        for x in range(2, 372):
            src = "https://www.newskj.org/news/game/index_"+str(x)+".html"
            print(src)
            yield scrapy.Request(url=src, callback=self.parse)

    def getcontet(self,response):

        description = response.xpath('normalize-space(//meta[@name="description"]/@content)').extract_first('')
        print("description---------------------------------------------"+description)
        fo = open("description.txt", "a", newline=None, encoding="utf-8")
        if description!="":
            fo.write(description + "\n")
